package com.atguigu.practice;

import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;
import scala.collection.mutable.HashSet;

public class Flink02_WordDistinct {

    public static void main(String[] args) throws Exception {

        //1.获取执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
//        env.setParallelism(1);

        //2.从端口读取数据
        DataStreamSource<String> textStream = env.socketTextStream("hadoop102", 9999);

        //3.压平
        SingleOutputStreamOperator<String> wordDS = textStream.flatMap(new FlatMapFunction<String, String>() {
            @Override
            public void flatMap(String value, Collector<String> out) throws Exception {
                String[] words = value.split(" ");
                for (String word : words) {
                    out.collect(word);
                }
            }
        });

        //4.去重
        SingleOutputStreamOperator<String> filter = wordDS
                .keyBy(data -> data)
                .filter(new FilterFunction<String>() {
                    HashSet<String> wordSet = new HashSet<>();

                    @Override
                    public boolean filter(String value) throws Exception {

                        boolean contains = wordSet.contains(value);

                        if (contains) {
                            return false;
                        } else {
                            wordSet.add(value);
                            return true;
                        }
                    }
                });

        //5.打印
        filter.print();

        //6.启动任务
        env.execute();

    }

}
